# -*- coding: utf-8 -*-

import glob
import codecs
import string
import re
import d15n_routines as r
from dm_surname_cz import *

# binary end of line to be used
n = '\x0D\x0A'

d = open('#cleanup_hb2_dropped_lines.dat', 'wb')
t = open('#cleanup_hb2_titles_replaced.dat', 'wb')

o = open('_cleanup_POC_ID_hb2.csv', 'wb')
for f in glob.glob('POC_ID*.csv'):
    counters = {'cnt_all_lines':0, 'cnt_titles_repair':0}
    if f not in (  'POC_ID_accept_login_v3.csv'
                 , 'POC_ID_acquitance_drawer_name_v3.csv'
                 , 'POC_ID_broker_sales_agent_v3.csv'
                 , 'POC_ID_creator_v4.csv'
                 , 'POC_ID_risk_manager_v3.csv'):
        pass
    else:
        print f
        i = open(f, 'rb')

        counters['cnt_all_lines'] = 0
        for l in i.readlines():
            counters['cnt_all_lines'] += 1
            l = r.remove_eol(l)

            if len(l) == 0:
                # skip empty rows
                d.write(f + ' -- ' + l + n)
            elif counters['cnt_all_lines'] == 1: # skip headers
                d.write(f + ' -- ' + l + n)
            else:
                # remove comas, brackets, numbers
                l = re.sub('[(),]', ' ', l)
                # reduce multiple spaces to one
                l = re.sub(' {2,}', ' ', l)
                l = r.ascii_upper(l)
                l = l.strip()
                cols = string.split(l, '|')
                
                id = cols[0]; nsm = cols[1]
                id = id.strip(); nsm = nsm.strip()

                # report short/long names/surnames
                #if len(nm) > 20 or len(sm) > 20: # too long sur/names
                #if len(nm) < 3 or len(sm) < 3: # too short sur/names
                #    d.write(f + ' -- ' + l + n)

                if nsm in (''): print l,

                nsm2 = r.remove_titles(nsm)
                nsm2 = nsm2.strip()
                if nsm2 <> nsm:
                    t.write(f + '\t>\t' + l + '\t>\t' + nsm2 + n)
                    counters['cnt_titles_repair'] += 1
                    nsm = nsm2[:]

                res = "%s\t%s\t%s" % (f, id, nsm)

                o.write(res + n)
        i.close()
        print f
        print "\tcounters['cnt_all_lines']", str(counters['cnt_all_lines'])
        print "\tcounters['cnt_titles_repair']", str(counters['cnt_titles_repair'])
o.close(); d.close(); t.close()
